{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型融合&&trick\n",
    "我们比赛中使用的stacking模型结构如下图所示\n",
    "\n",
    "  ![img](../img/stacking.png)\n",
    " \n",
    "### Snapshot Emsemble\n",
    "   在stacking第二层模型中我们还加入了深度融合的方法，[论文地址](https://arxiv.org/abs/1704.00109)\n",
    "   \n",
    "### Pesudo Labeling\n",
    "   我们使用的另外一个trick就是pesudo-labeling 方法，它适用于所有给定测试集的比赛 [教程](https://shaoanlu.wordpress.com/2017/04/10/a-simple-pseudo-labeling-function-implementation-in-keras/)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入相应的包\n",
    "import pickle\n",
    "import glob\n",
    "from config import Config\n",
    "from keras.utils import np_utils\n",
    "from keras.layers import *\n",
    "from model.snapshot import SnapshotCallbackBuilder\n",
    "from keras.models import *\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_X = '../data/All_cut_train_text.txt'\n",
    "TEXT_X = '../data/' + 'News_cut_test_text.txt'\n",
    "config = Config()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 准备基本特征和OOF文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_prepare():\n",
    "    oof_filename = []\n",
    "    test_filename = []\n",
    "\n",
    "    \n",
    "\n",
    "    # load text feature\n",
    "    train_y = []\n",
    "\n",
    "\n",
    "\n",
    "    with open(TRAIN_X, 'r', encoding='utf-8') as f:\n",
    "        lines = f.readlines()\n",
    "        for line in lines:\n",
    "            line = line.strip()\n",
    "            line = line.split('\\t')\n",
    "            label = int(line[2])\n",
    "            train_y.append(label)\n",
    "\n",
    "    with open(config.FEATURES_test_FILE, 'rb') as f:\n",
    "        test_features = pickle.load(f)\n",
    "    with open(config.FEATURES_FILE, 'rb') as f:\n",
    "        features = pickle.load(f)\n",
    "\n",
    "    with open(config.OCR_FEATURES_test_FILE, 'rb') as f:\n",
    "        ocr_test_features = pickle.load(f)\n",
    "    with open(config.OCR_FEATURES_FILE, 'rb') as f:\n",
    "        ocr_features = pickle.load(f)\n",
    "        \n",
    "    scaler = MinMaxScaler()\n",
    "    all_feature = np.concatenate([features, test_features, ocr_test_features, ocr_features], axis=0)\n",
    "    scaler.fit(all_feature)\n",
    "    features = scaler.transform(features)\n",
    "    test_features = scaler.transform(test_features)\n",
    "    ocr_features = scaler.transform(ocr_features)\n",
    "    ocr_test_features = scaler.transform(ocr_test_features)\n",
    "\n",
    "    train_y = np_utils.to_categorical(train_y)\n",
    "\n",
    "    with open('../data/train_x_250.pkl', 'rb') as f:\n",
    "        train_x = pickle.load(f)\n",
    "\n",
    "    with open('../data/' + 'test_x_250.pkl', 'rb') as f:\n",
    "        test_x = pickle.load(f)\n",
    "        \n",
    "    # 联合OCR提取的特征\n",
    "    with open('../data/ocr_train_x_250.pkl', 'rb') as f:\n",
    "        train_ocr_x = pickle.load(f)\n",
    "\n",
    "    with open('../data/ocr_test_x_250.pkl', 'rb') as f:\n",
    "        test_ocr_x = pickle.load(f)\n",
    "        \n",
    "    # load oof train and oof test\n",
    "    filenames = glob.glob('../data/result/*oof*')\n",
    "    for filename in filenames:\n",
    "        oof_filename.append(filename)\n",
    "        test_filename.append(filename.replace('_oof_', '_oof_'))\n",
    "\n",
    "    oof_data = []\n",
    "    test_data = []\n",
    "\n",
    "    for tra, tes in zip(oof_filename, test_filename):\n",
    "        with open(tra, 'rb') as f:\n",
    "            oof_data.extend(pickle.load(f)[:len(train_x)])\n",
    "        with open(tes, 'rb') as f:\n",
    "            test_data.extend(pickle.load(f)[:len(test_x)])\n",
    "            \n",
    "    train_x = np.concatenate((train_x, train_ocr_x, features, ocr_features, oof_data[:len(train_x)]), axis=-1)\n",
    "    test_x = np.concatenate((test_x, test_ocr_x, test_features, ocr_test_features, test_data[:len(test_x)]), axis=-1)\n",
    "\n",
    "    train = {}\n",
    "    test = {}\n",
    "    train = train_x\n",
    "    test  = test_x\n",
    "    return train, train_y, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    }
   ],
   "source": [
    "train, train_y, test = data_prepare()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 这里只是使用了简单的DNN来做模型stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(train_x):\n",
    "    input_shape = Input(shape=(train_x.shape[1],), name='news')\n",
    "    x = Dense(256, activation='relu')(input_shape)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Dense(3, activation=\"softmax\")(x)\n",
    "    res_model = Model(inputs=[input_shape], outputs=x)\n",
    "    return res_model\n",
    "\n",
    "def check_accuracy(pred, label, test_index):\n",
    "    right = 0\n",
    "    total = 0\n",
    "    for count, re in enumerate(pred):\n",
    "        cc = test_index[count]\n",
    "        if cc >= 48480:\n",
    "            continue\n",
    "        total += 1\n",
    "        flag = np.argmax(re)\n",
    "        if int(flag) == int(np.argmax(label[count])):\n",
    "            right += 1\n",
    "    return right / total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 准备stacking模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 第一次stacking\n",
    "def stacking_first(train, train_y, test):\n",
    "    savepath = './stack_/'\n",
    "    if not os.path.exists(savepath):\n",
    "        os.mkdir(savepath)\n",
    "    count_kflod = 0\n",
    "    num_folds = 6\n",
    "    kf = KFold(n_splits=num_folds, shuffle=True, random_state=10)\n",
    "    predict = np.zeros((test.shape[0], 3))\n",
    "    oof_predict = np.zeros((train.shape[0], 3))\n",
    "    scores = []\n",
    "    for train_index, test_index in kf.split(train):\n",
    "\n",
    "        kfold_X_train = {}\n",
    "        kfold_X_valid = {}\n",
    "\n",
    "        y_train, y_test = train_y[train_index], train_y[test_index]\n",
    "\n",
    "        kfold_X_train, kfold_X_valid = train[train_index], train[test_index]\n",
    "\n",
    "        model_prefix = savepath + 'DNN' + str(count_kflod)\n",
    "        if not os.path.exists(model_prefix):\n",
    "            os.mkdir(model_prefix)\n",
    "\n",
    "        M = 4  # number of snapshots\n",
    "        alpha_zero = 1e-3  # initial learning rate\n",
    "        snap_epoch = 16\n",
    "        snapshot = SnapshotCallbackBuilder(snap_epoch, M, alpha_zero)\n",
    "\n",
    "        res_model = get_model(train)\n",
    "        res_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "        # res_model.fit(train_x, train_y, batch_size=BATCH_SIZE, epochs=EPOCH, verbose=1,  class_weight=class_weight)\n",
    "        res_model.fit(kfold_X_train, y_train, batch_size=BATCH_SIZE, epochs=snap_epoch, verbose=1,\n",
    "                      validation_data=(kfold_X_valid, y_test),\n",
    "                      callbacks=snapshot.get_callbacks(model_save_place=model_prefix))\n",
    "\n",
    "        evaluations = []\n",
    "        for i in os.listdir(model_prefix):\n",
    "            if '.h5' in i:\n",
    "                evaluations.append(i)\n",
    "\n",
    "        preds1 = np.zeros((test.shape[0], 3))\n",
    "        preds2 = np.zeros((len(kfold_X_valid), 3))\n",
    "        for run, i in enumerate(evaluations):\n",
    "            res_model.load_weights(os.path.join(model_prefix, i))\n",
    "            preds1 += res_model.predict(test, verbose=1) / len(evaluations)\n",
    "            preds2 += res_model.predict(kfold_X_valid, batch_size=128) / len(evaluations)\n",
    "\n",
    "        predict += preds1 / num_folds\n",
    "        oof_predict[test_index] = preds2\n",
    "\n",
    "        accuracy = check_accuracy(oof_predict[test_index], y_test, test_index)\n",
    "        print('the kflod cv is : ', str(accuracy))\n",
    "        count_kflod += 1\n",
    "        scores.append(accuracy)\n",
    "    print('total scores is ', np.mean(scores))\n",
    "    return predict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40400 samples, validate on 8080 samples\n",
      "Epoch 1/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6914 - acc: 0.7282 - val_loss: 0.6503 - val_acc: 0.7384\n",
      "Epoch 2/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.6501 - acc: 0.7364 - val_loss: 0.6455 - val_acc: 0.7381\n",
      "Epoch 3/16\n",
      "40400/40400 [==============================] - 2s 55us/step - loss: 0.6369 - acc: 0.7383 - val_loss: 0.6455 - val_acc: 0.7391\n",
      "Epoch 4/16\n",
      "40400/40400 [==============================] - 2s 51us/step - loss: 0.6288 - acc: 0.7410 - val_loss: 0.6453 - val_acc: 0.7396\n",
      "Epoch 5/16\n",
      "40400/40400 [==============================] - 2s 53us/step - loss: 0.6341 - acc: 0.7388 - val_loss: 0.6480 - val_acc: 0.7405\n",
      "Epoch 6/16\n",
      "40400/40400 [==============================] - 2s 52us/step - loss: 0.6283 - acc: 0.7400 - val_loss: 0.6465 - val_acc: 0.7405\n",
      "Epoch 7/16\n",
      "40400/40400 [==============================] - 2s 54us/step - loss: 0.6191 - acc: 0.7424 - val_loss: 0.6458 - val_acc: 0.7413\n",
      "Epoch 8/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6079 - acc: 0.7474 - val_loss: 0.6474 - val_acc: 0.7407\n",
      "Epoch 9/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.6177 - acc: 0.7461 - val_loss: 0.6448 - val_acc: 0.7384\n",
      "Epoch 10/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6072 - acc: 0.7486 - val_loss: 0.6464 - val_acc: 0.7368\n",
      "Epoch 11/16\n",
      "40400/40400 [==============================] - 2s 55us/step - loss: 0.5922 - acc: 0.7546 - val_loss: 0.6492 - val_acc: 0.7382\n",
      "Epoch 12/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.5824 - acc: 0.7584 - val_loss: 0.6517 - val_acc: 0.7390\n",
      "Epoch 13/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.5952 - acc: 0.7544 - val_loss: 0.6536 - val_acc: 0.7385\n",
      "Epoch 14/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5833 - acc: 0.7594 - val_loss: 0.6563 - val_acc: 0.7354\n",
      "Epoch 15/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5686 - acc: 0.7645 - val_loss: 0.6571 - val_acc: 0.7340\n",
      "Epoch 16/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.5507 - acc: 0.7730 - val_loss: 0.6632 - val_acc: 0.7351\n",
      "35/35 [==============================] - 0s 702us/step\n",
      "35/35 [==============================] - 0s 53us/step\n",
      "35/35 [==============================] - 0s 54us/step\n",
      "35/35 [==============================] - 0s 51us/step\n",
      "the kflod cv is :  0.7391089108910891\n",
      "Train on 40400 samples, validate on 8080 samples\n",
      "Epoch 1/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.6921 - acc: 0.7265 - val_loss: 0.6423 - val_acc: 0.7396\n",
      "Epoch 2/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6519 - acc: 0.7358 - val_loss: 0.6357 - val_acc: 0.7399\n",
      "Epoch 3/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.6395 - acc: 0.7375 - val_loss: 0.6336 - val_acc: 0.7391\n",
      "Epoch 4/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6300 - acc: 0.7400 - val_loss: 0.6321 - val_acc: 0.7395\n",
      "Epoch 5/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6375 - acc: 0.7384 - val_loss: 0.6375 - val_acc: 0.7376\n",
      "Epoch 6/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6290 - acc: 0.7412 - val_loss: 0.6327 - val_acc: 0.7397\n",
      "Epoch 7/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.6193 - acc: 0.7435 - val_loss: 0.6336 - val_acc: 0.7387\n",
      "Epoch 8/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.6096 - acc: 0.7469 - val_loss: 0.6333 - val_acc: 0.7399\n",
      "Epoch 9/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6190 - acc: 0.7442 - val_loss: 0.6366 - val_acc: 0.7402\n",
      "Epoch 10/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6104 - acc: 0.7482 - val_loss: 0.6372 - val_acc: 0.7381\n",
      "Epoch 11/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.5949 - acc: 0.7540 - val_loss: 0.6365 - val_acc: 0.7377\n",
      "Epoch 12/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.5814 - acc: 0.7575 - val_loss: 0.6410 - val_acc: 0.7386\n",
      "Epoch 13/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.5935 - acc: 0.7552 - val_loss: 0.6420 - val_acc: 0.7379\n",
      "Epoch 14/16\n",
      "40400/40400 [==============================] - 2s 60us/step - loss: 0.5835 - acc: 0.7581 - val_loss: 0.6428 - val_acc: 0.7361\n",
      "Epoch 15/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5675 - acc: 0.7648 - val_loss: 0.6456 - val_acc: 0.7358\n",
      "Epoch 16/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5519 - acc: 0.7709 - val_loss: 0.6491 - val_acc: 0.7348\n",
      "35/35 [==============================] - 0s 1ms/step\n",
      "35/35 [==============================] - 0s 53us/step\n",
      "35/35 [==============================] - 0s 56us/step\n",
      "35/35 [==============================] - 0s 64us/step\n",
      "the kflod cv is :  0.7397277227722773\n",
      "Train on 40400 samples, validate on 8080 samples\n",
      "Epoch 1/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.6931 - acc: 0.7293 - val_loss: 0.6450 - val_acc: 0.7318\n",
      "Epoch 2/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6521 - acc: 0.7361 - val_loss: 0.6382 - val_acc: 0.7308\n",
      "Epoch 3/16\n",
      "40400/40400 [==============================] - 2s 60us/step - loss: 0.6394 - acc: 0.7389 - val_loss: 0.6355 - val_acc: 0.7318\n",
      "Epoch 4/16\n",
      "40400/40400 [==============================] - 2s 61us/step - loss: 0.6311 - acc: 0.7407 - val_loss: 0.6349 - val_acc: 0.7327\n",
      "Epoch 5/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.6367 - acc: 0.7403 - val_loss: 0.6369 - val_acc: 0.7316\n",
      "Epoch 6/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.6298 - acc: 0.7411 - val_loss: 0.6354 - val_acc: 0.7311\n",
      "Epoch 7/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6200 - acc: 0.7463 - val_loss: 0.6353 - val_acc: 0.7323\n",
      "Epoch 8/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.6086 - acc: 0.7494 - val_loss: 0.6358 - val_acc: 0.7323\n",
      "Epoch 9/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.6176 - acc: 0.7473 - val_loss: 0.6369 - val_acc: 0.7342\n",
      "Epoch 10/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6086 - acc: 0.7509 - val_loss: 0.6398 - val_acc: 0.7313\n",
      "Epoch 11/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.5954 - acc: 0.7542 - val_loss: 0.6396 - val_acc: 0.7296\n",
      "Epoch 12/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.5819 - acc: 0.7614 - val_loss: 0.6396 - val_acc: 0.7302\n",
      "Epoch 13/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5964 - acc: 0.7546 - val_loss: 0.6409 - val_acc: 0.7285\n",
      "Epoch 14/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5856 - acc: 0.7574 - val_loss: 0.6438 - val_acc: 0.7275\n",
      "Epoch 15/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.5671 - acc: 0.7671 - val_loss: 0.6536 - val_acc: 0.7307\n",
      "Epoch 16/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.5530 - acc: 0.7712 - val_loss: 0.6498 - val_acc: 0.7297\n",
      "35/35 [==============================] - 0s 1ms/step\n",
      "35/35 [==============================] - 0s 53us/step\n",
      "35/35 [==============================] - 0s 54us/step\n",
      "35/35 [==============================] - 0s 57us/step\n",
      "the kflod cv is :  0.7320544554455446\n",
      "Train on 40400 samples, validate on 8080 samples\n",
      "Epoch 1/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6939 - acc: 0.7303 - val_loss: 0.6474 - val_acc: 0.7312\n",
      "Epoch 2/16\n",
      "40400/40400 [==============================] - 2s 54us/step - loss: 0.6521 - acc: 0.7348 - val_loss: 0.6421 - val_acc: 0.7334\n",
      "Epoch 3/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6402 - acc: 0.7384 - val_loss: 0.6365 - val_acc: 0.7356\n",
      "Epoch 4/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.6301 - acc: 0.7417 - val_loss: 0.6359 - val_acc: 0.7354\n",
      "Epoch 5/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6367 - acc: 0.7398 - val_loss: 0.6441 - val_acc: 0.7354\n",
      "Epoch 6/16\n",
      "40400/40400 [==============================] - 2s 54us/step - loss: 0.6276 - acc: 0.7420 - val_loss: 0.6395 - val_acc: 0.7353\n",
      "Epoch 7/16\n",
      "40400/40400 [==============================] - 2s 54us/step - loss: 0.6200 - acc: 0.7444 - val_loss: 0.6349 - val_acc: 0.7337\n",
      "Epoch 8/16\n",
      "40400/40400 [==============================] - 2s 51us/step - loss: 0.6099 - acc: 0.7471 - val_loss: 0.6357 - val_acc: 0.7332\n",
      "Epoch 9/16\n",
      "40400/40400 [==============================] - 2s 52us/step - loss: 0.6159 - acc: 0.7449 - val_loss: 0.6395 - val_acc: 0.7342\n",
      "Epoch 10/16\n",
      "40400/40400 [==============================] - 2s 54us/step - loss: 0.6083 - acc: 0.7499 - val_loss: 0.6379 - val_acc: 0.7351\n",
      "Epoch 11/16\n",
      "40400/40400 [==============================] - 2s 55us/step - loss: 0.5947 - acc: 0.7546 - val_loss: 0.6388 - val_acc: 0.7332\n",
      "Epoch 12/16\n",
      "40400/40400 [==============================] - 2s 55us/step - loss: 0.5812 - acc: 0.7599 - val_loss: 0.6414 - val_acc: 0.7335\n",
      "Epoch 13/16\n",
      "40400/40400 [==============================] - 2s 53us/step - loss: 0.5938 - acc: 0.7564 - val_loss: 0.6416 - val_acc: 0.7327\n",
      "Epoch 14/16\n",
      "40400/40400 [==============================] - 2s 52us/step - loss: 0.5849 - acc: 0.7598 - val_loss: 0.6437 - val_acc: 0.7323\n",
      "Epoch 15/16\n",
      "40400/40400 [==============================] - 2s 55us/step - loss: 0.5663 - acc: 0.7648 - val_loss: 0.6537 - val_acc: 0.7313\n",
      "Epoch 16/16\n",
      "40400/40400 [==============================] - 2s 52us/step - loss: 0.5529 - acc: 0.7722 - val_loss: 0.6494 - val_acc: 0.7312\n",
      "35/35 [==============================] - 0s 2ms/step\n",
      "35/35 [==============================] - 0s 55us/step\n",
      "35/35 [==============================] - 0s 53us/step\n",
      "35/35 [==============================] - 0s 55us/step\n",
      "the kflod cv is :  0.7342821782178218\n",
      "Train on 40400 samples, validate on 8080 samples\n",
      "Epoch 1/16\n",
      "40400/40400 [==============================] - 2s 53us/step - loss: 0.6908 - acc: 0.7296 - val_loss: 0.6604 - val_acc: 0.7333\n",
      "Epoch 2/16\n",
      "40400/40400 [==============================] - 2s 50us/step - loss: 0.6495 - acc: 0.7363 - val_loss: 0.6514 - val_acc: 0.7342\n",
      "Epoch 3/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6362 - acc: 0.7380 - val_loss: 0.6510 - val_acc: 0.7354\n",
      "Epoch 4/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.6259 - acc: 0.7400 - val_loss: 0.6511 - val_acc: 0.7350\n",
      "Epoch 5/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6324 - acc: 0.7391 - val_loss: 0.6540 - val_acc: 0.7335\n",
      "Epoch 6/16\n",
      "40400/40400 [==============================] - 2s 54us/step - loss: 0.6257 - acc: 0.7409 - val_loss: 0.6530 - val_acc: 0.7332\n",
      "Epoch 7/16\n",
      "40400/40400 [==============================] - 2s 54us/step - loss: 0.6134 - acc: 0.7436 - val_loss: 0.6570 - val_acc: 0.7313\n",
      "Epoch 8/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.6038 - acc: 0.7467 - val_loss: 0.6540 - val_acc: 0.7319\n",
      "Epoch 9/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.6143 - acc: 0.7469 - val_loss: 0.6561 - val_acc: 0.7323\n",
      "Epoch 10/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6047 - acc: 0.7493 - val_loss: 0.6568 - val_acc: 0.7325\n",
      "Epoch 11/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.5919 - acc: 0.7522 - val_loss: 0.6586 - val_acc: 0.7317\n",
      "Epoch 12/16\n",
      "40400/40400 [==============================] - 2s 55us/step - loss: 0.5789 - acc: 0.7573 - val_loss: 0.6611 - val_acc: 0.7319\n",
      "Epoch 13/16\n",
      "40400/40400 [==============================] - 2s 54us/step - loss: 0.5918 - acc: 0.7548 - val_loss: 0.6572 - val_acc: 0.7306\n",
      "Epoch 14/16\n",
      "40400/40400 [==============================] - 2s 55us/step - loss: 0.5814 - acc: 0.7581 - val_loss: 0.6617 - val_acc: 0.7325\n",
      "Epoch 15/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5632 - acc: 0.7655 - val_loss: 0.6722 - val_acc: 0.7282\n",
      "Epoch 16/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5488 - acc: 0.7701 - val_loss: 0.6735 - val_acc: 0.7290\n",
      "35/35 [==============================] - 0s 2ms/step\n",
      "35/35 [==============================] - 0s 54us/step\n",
      "35/35 [==============================] - 0s 58us/step\n",
      "35/35 [==============================] - 0s 55us/step\n",
      "the kflod cv is :  0.7325495049504951\n",
      "Train on 40400 samples, validate on 8080 samples\n",
      "Epoch 1/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6954 - acc: 0.7297 - val_loss: 0.6405 - val_acc: 0.7408\n",
      "Epoch 2/16\n",
      "40400/40400 [==============================] - 2s 55us/step - loss: 0.6522 - acc: 0.7360 - val_loss: 0.6331 - val_acc: 0.7415\n",
      "Epoch 3/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6408 - acc: 0.7362 - val_loss: 0.6319 - val_acc: 0.7413\n",
      "Epoch 4/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.6318 - acc: 0.7379 - val_loss: 0.6319 - val_acc: 0.7411\n",
      "Epoch 5/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.6371 - acc: 0.7362 - val_loss: 0.6327 - val_acc: 0.7394\n",
      "Epoch 6/16\n",
      "40400/40400 [==============================] - 2s 54us/step - loss: 0.6294 - acc: 0.7406 - val_loss: 0.6343 - val_acc: 0.7399\n",
      "Epoch 7/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.6197 - acc: 0.7417 - val_loss: 0.6321 - val_acc: 0.7397\n",
      "Epoch 8/16\n",
      "40400/40400 [==============================] - 2s 62us/step - loss: 0.6104 - acc: 0.7442 - val_loss: 0.6330 - val_acc: 0.7412\n",
      "Epoch 9/16\n",
      "40400/40400 [==============================] - 2s 60us/step - loss: 0.6192 - acc: 0.7459 - val_loss: 0.6343 - val_acc: 0.7392\n",
      "Epoch 10/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.6112 - acc: 0.7477 - val_loss: 0.6351 - val_acc: 0.7403\n",
      "Epoch 11/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5973 - acc: 0.7521 - val_loss: 0.6351 - val_acc: 0.7391\n",
      "Epoch 12/16\n",
      "40400/40400 [==============================] - 2s 59us/step - loss: 0.5838 - acc: 0.7569 - val_loss: 0.6364 - val_acc: 0.7391\n",
      "Epoch 13/16\n",
      "40400/40400 [==============================] - 2s 56us/step - loss: 0.5950 - acc: 0.7537 - val_loss: 0.6407 - val_acc: 0.7373\n",
      "Epoch 14/16\n",
      "40400/40400 [==============================] - 2s 57us/step - loss: 0.5855 - acc: 0.7571 - val_loss: 0.6437 - val_acc: 0.7369\n",
      "Epoch 15/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.5676 - acc: 0.7665 - val_loss: 0.6450 - val_acc: 0.7376\n",
      "Epoch 16/16\n",
      "40400/40400 [==============================] - 2s 58us/step - loss: 0.5540 - acc: 0.7702 - val_loss: 0.6465 - val_acc: 0.7369\n",
      "35/35 [==============================] - 0s 2ms/step\n",
      "35/35 [==============================] - 0s 54us/step\n",
      "35/35 [==============================] - 0s 59us/step\n",
      "35/35 [==============================] - 0s 56us/step\n",
      "the kflod cv is :  0.7423267326732673\n",
      "total scores is  0.7366749174917492\n"
     ]
    }
   ],
   "source": [
    "predicts = stacking_first(train, train_y, test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 这里使用pesudo-labeling方法\n",
    "具体思路如下图所示\n",
    "![img](../img/pesudo.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用pseudo-labeling做第二次stacking\n",
    "def stacking_pseudo(train, train_y, test, results):\n",
    "    answer = np.zeros((results.shape[0], 1))\n",
    "    for count in range(len(results)):\n",
    "        answer[count] = np.argmax(results[count])\n",
    "    answer = np_utils.to_categorical(answer)\n",
    "    train_y = np.concatenate([train_y, answer], axis=0)\n",
    "    train = np.concatenate([train, test], axis=0)\n",
    "\n",
    "\n",
    "    savepath = './pesudo_/'\n",
    "    if not os.path.exists(savepath):\n",
    "        os.mkdir(savepath)\n",
    "    count_kflod = 0\n",
    "    num_folds = 6\n",
    "    kf = KFold(n_splits=num_folds, shuffle=True, random_state=10)\n",
    "    predict = np.zeros((test.shape[0], 3))\n",
    "    oof_predict = np.zeros((train.shape[0], 3))\n",
    "    scores = []\n",
    "    for train_index, test_index in kf.split(train):\n",
    "\n",
    "        kfold_X_train = {}\n",
    "        kfold_X_valid = {}\n",
    "\n",
    "        y_train, y_test = train_y[train_index], train_y[test_index]\n",
    "\n",
    "        kfold_X_train, kfold_X_valid = train[train_index], train[test_index]\n",
    "\n",
    "        model_prefix = savepath + 'DNN' + str(count_kflod)\n",
    "        if not os.path.exists(model_prefix):\n",
    "            os.mkdir(model_prefix)\n",
    "\n",
    "        M = 4  # number of snapshots\n",
    "        alpha_zero = 1e-3  # initial learning rate\n",
    "        snap_epoch = 16\n",
    "        snapshot = SnapshotCallbackBuilder(snap_epoch, M, alpha_zero)\n",
    "\n",
    "        res_model = get_model(train)\n",
    "        res_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "        # res_model.fit(train_x, train_y, batch_size=BATCH_SIZE, epochs=EPOCH, verbose=1,  class_weight=class_weight)\n",
    "        res_model.fit(kfold_X_train, y_train, batch_size=BATCH_SIZE, epochs=snap_epoch, verbose=1,\n",
    "                      validation_data=(kfold_X_valid, y_test),\n",
    "                      callbacks=snapshot.get_callbacks(model_save_place=model_prefix))\n",
    "\n",
    "        evaluations = []\n",
    "        for i in os.listdir(model_prefix):\n",
    "            if '.h5' in i:\n",
    "                evaluations.append(i)\n",
    "        print(evaluations)\n",
    "\n",
    "        preds1 = np.zeros((test.shape[0], 3))\n",
    "        preds2 = np.zeros((len(kfold_X_valid), 3))\n",
    "        for run, i in enumerate(evaluations):\n",
    "            res_model.load_weights(os.path.join(model_prefix, i))\n",
    "            preds1 += res_model.predict(test, verbose=1) / len(evaluations)\n",
    "            preds2 += res_model.predict(kfold_X_valid, batch_size=128) / len(evaluations)\n",
    "\n",
    "        predict += preds1 / num_folds\n",
    "        oof_predict[test_index] = preds2\n",
    "\n",
    "        accuracy = check_accuracy(oof_predict[test_index], y_test, test_index)\n",
    "        print('the kflod cv is : ', str(accuracy))\n",
    "        count_kflod += 1\n",
    "        scores.append(accuracy)\n",
    "    print('total scores is ', np.mean(scores))\n",
    "    return predict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40429 samples, validate on 8086 samples\n",
      "Epoch 1/16\n",
      "40429/40429 [==============================] - 3s 63us/step - loss: 0.6948 - acc: 0.7303 - val_loss: 0.6348 - val_acc: 0.7420\n",
      "Epoch 2/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.6545 - acc: 0.7349 - val_loss: 0.6271 - val_acc: 0.7423\n",
      "Epoch 3/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.6406 - acc: 0.7368 - val_loss: 0.6270 - val_acc: 0.7433\n",
      "Epoch 4/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.6314 - acc: 0.7381 - val_loss: 0.6254 - val_acc: 0.7446\n",
      "Epoch 5/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.6387 - acc: 0.7375 - val_loss: 0.6243 - val_acc: 0.7441\n",
      "Epoch 6/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.6294 - acc: 0.7390 - val_loss: 0.6257 - val_acc: 0.7451\n",
      "Epoch 7/16\n",
      "40429/40429 [==============================] - 2s 53us/step - loss: 0.6180 - acc: 0.7444 - val_loss: 0.6246 - val_acc: 0.7459\n",
      "Epoch 8/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.6108 - acc: 0.7466 - val_loss: 0.6237 - val_acc: 0.7476\n",
      "Epoch 9/16\n",
      "40429/40429 [==============================] - 2s 59us/step - loss: 0.6188 - acc: 0.7446 - val_loss: 0.6248 - val_acc: 0.7456\n",
      "Epoch 10/16\n",
      "40429/40429 [==============================] - 2s 58us/step - loss: 0.6116 - acc: 0.7461 - val_loss: 0.6232 - val_acc: 0.7455\n",
      "Epoch 11/16\n",
      "40429/40429 [==============================] - 2s 59us/step - loss: 0.5959 - acc: 0.7529 - val_loss: 0.6243 - val_acc: 0.7435\n",
      "Epoch 12/16\n",
      "40429/40429 [==============================] - 2s 58us/step - loss: 0.5819 - acc: 0.7581 - val_loss: 0.6257 - val_acc: 0.7452\n",
      "Epoch 13/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.5964 - acc: 0.7514 - val_loss: 0.6259 - val_acc: 0.7451\n",
      "Epoch 14/16\n",
      "40429/40429 [==============================] - 2s 59us/step - loss: 0.5861 - acc: 0.7581 - val_loss: 0.6271 - val_acc: 0.7430\n",
      "Epoch 15/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.5685 - acc: 0.7622 - val_loss: 0.6337 - val_acc: 0.7431\n",
      "Epoch 16/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.5529 - acc: 0.7721 - val_loss: 0.6337 - val_acc: 0.7433\n",
      "['Model-1.h5', 'Model-4.h5', 'Model-3.h5', 'Model-2.h5']\n",
      "35/35 [==============================] - 0s 3ms/step\n",
      "35/35 [==============================] - 0s 56us/step\n",
      "35/35 [==============================] - 0s 54us/step\n",
      "35/35 [==============================] - 0s 51us/step\n",
      "the kflod cv is :  0.744988864142539\n",
      "Train on 40429 samples, validate on 8086 samples\n",
      "Epoch 1/16\n",
      "40429/40429 [==============================] - 2s 61us/step - loss: 0.6929 - acc: 0.7319 - val_loss: 0.6611 - val_acc: 0.7323\n",
      "Epoch 2/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.6495 - acc: 0.7384 - val_loss: 0.6526 - val_acc: 0.7326\n",
      "Epoch 3/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.6356 - acc: 0.7404 - val_loss: 0.6517 - val_acc: 0.7331\n",
      "Epoch 4/16\n",
      "40429/40429 [==============================] - 3s 63us/step - loss: 0.6269 - acc: 0.7406 - val_loss: 0.6493 - val_acc: 0.7323\n",
      "Epoch 5/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.6340 - acc: 0.7406 - val_loss: 0.6586 - val_acc: 0.7326\n",
      "Epoch 6/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.6253 - acc: 0.7435 - val_loss: 0.6504 - val_acc: 0.7324\n",
      "Epoch 7/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.6163 - acc: 0.7450 - val_loss: 0.6501 - val_acc: 0.7320\n",
      "Epoch 8/16\n",
      "40429/40429 [==============================] - 2s 59us/step - loss: 0.6061 - acc: 0.7475 - val_loss: 0.6519 - val_acc: 0.7324\n",
      "Epoch 9/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.6139 - acc: 0.7456 - val_loss: 0.6543 - val_acc: 0.7309\n",
      "Epoch 10/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.6066 - acc: 0.7494 - val_loss: 0.6516 - val_acc: 0.7339\n",
      "Epoch 11/16\n",
      "40429/40429 [==============================] - 2s 55us/step - loss: 0.5936 - acc: 0.7545 - val_loss: 0.6571 - val_acc: 0.7334\n",
      "Epoch 12/16\n",
      "40429/40429 [==============================] - 2s 55us/step - loss: 0.5780 - acc: 0.7594 - val_loss: 0.6576 - val_acc: 0.7332\n",
      "Epoch 13/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.5926 - acc: 0.7553 - val_loss: 0.6570 - val_acc: 0.7306\n",
      "Epoch 14/16\n",
      "40429/40429 [==============================] - 2s 59us/step - loss: 0.5812 - acc: 0.7585 - val_loss: 0.6631 - val_acc: 0.7311\n",
      "Epoch 15/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.5634 - acc: 0.7664 - val_loss: 0.6730 - val_acc: 0.7303\n",
      "Epoch 16/16\n",
      "40429/40429 [==============================] - 3s 63us/step - loss: 0.5457 - acc: 0.7720 - val_loss: 0.6706 - val_acc: 0.7306\n",
      "['Model-1.h5', 'Model-4.h5', 'Model-3.h5', 'Model-2.h5']\n",
      "35/35 [==============================] - 0s 3ms/step\n",
      "35/35 [==============================] - 0s 66us/step\n",
      "35/35 [==============================] - 0s 53us/step\n",
      "35/35 [==============================] - 0s 56us/step\n",
      "the kflod cv is :  0.7317405298341173\n",
      "Train on 40429 samples, validate on 8086 samples\n",
      "Epoch 1/16\n",
      "40429/40429 [==============================] - 3s 73us/step - loss: 0.6936 - acc: 0.7286 - val_loss: 0.6437 - val_acc: 0.7355\n",
      "Epoch 2/16\n",
      "40429/40429 [==============================] - 3s 65us/step - loss: 0.6516 - acc: 0.7356 - val_loss: 0.6343 - val_acc: 0.7347\n",
      "Epoch 3/16\n",
      "40429/40429 [==============================] - 3s 71us/step - loss: 0.6384 - acc: 0.7381 - val_loss: 0.6306 - val_acc: 0.7362\n",
      "Epoch 4/16\n",
      "40429/40429 [==============================] - 2s 62us/step - loss: 0.6311 - acc: 0.7403 - val_loss: 0.6296 - val_acc: 0.7357\n",
      "Epoch 5/16\n",
      "40429/40429 [==============================] - 3s 71us/step - loss: 0.6383 - acc: 0.7385 - val_loss: 0.6324 - val_acc: 0.7360\n",
      "Epoch 6/16\n",
      "40429/40429 [==============================] - 3s 63us/step - loss: 0.6288 - acc: 0.7412 - val_loss: 0.6298 - val_acc: 0.7361\n",
      "Epoch 7/16\n",
      "40429/40429 [==============================] - 3s 66us/step - loss: 0.6195 - acc: 0.7452 - val_loss: 0.6309 - val_acc: 0.7341\n",
      "Epoch 8/16\n",
      "40429/40429 [==============================] - 3s 65us/step - loss: 0.6098 - acc: 0.7487 - val_loss: 0.6297 - val_acc: 0.7342\n",
      "Epoch 9/16\n",
      "40429/40429 [==============================] - 3s 64us/step - loss: 0.6207 - acc: 0.7455 - val_loss: 0.6308 - val_acc: 0.7347\n",
      "Epoch 10/16\n",
      "40429/40429 [==============================] - 3s 63us/step - loss: 0.6103 - acc: 0.7496 - val_loss: 0.6369 - val_acc: 0.7373\n",
      "Epoch 11/16\n",
      "40429/40429 [==============================] - 3s 63us/step - loss: 0.5966 - acc: 0.7543 - val_loss: 0.6331 - val_acc: 0.7367\n",
      "Epoch 12/16\n",
      "40429/40429 [==============================] - 3s 62us/step - loss: 0.5838 - acc: 0.7605 - val_loss: 0.6343 - val_acc: 0.7353\n",
      "Epoch 13/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.5959 - acc: 0.7544 - val_loss: 0.6398 - val_acc: 0.7327\n",
      "Epoch 14/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.5854 - acc: 0.7595 - val_loss: 0.6361 - val_acc: 0.7351\n",
      "Epoch 15/16\n",
      "40429/40429 [==============================] - 2s 61us/step - loss: 0.5684 - acc: 0.7676 - val_loss: 0.6434 - val_acc: 0.7319\n",
      "Epoch 16/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.5531 - acc: 0.7725 - val_loss: 0.6461 - val_acc: 0.7310\n",
      "['Model-1.h5', 'Model-4.h5', 'Model-3.h5', 'Model-2.h5']\n",
      "35/35 [==============================] - 0s 3ms/step\n",
      "35/35 [==============================] - 0s 66us/step\n",
      "35/35 [==============================] - 0s 55us/step\n",
      "35/35 [==============================] - 0s 55us/step\n",
      "the kflod cv is :  0.7370961752692164\n",
      "Train on 40429 samples, validate on 8086 samples\n",
      "Epoch 1/16\n",
      "40429/40429 [==============================] - 2s 61us/step - loss: 0.6908 - acc: 0.7297 - val_loss: 0.6519 - val_acc: 0.7319\n",
      "Epoch 2/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.6492 - acc: 0.7361 - val_loss: 0.6458 - val_acc: 0.7334\n",
      "Epoch 3/16\n",
      "40429/40429 [==============================] - 2s 59us/step - loss: 0.6371 - acc: 0.7377 - val_loss: 0.6440 - val_acc: 0.7352\n",
      "Epoch 4/16\n",
      "40429/40429 [==============================] - 3s 68us/step - loss: 0.6284 - acc: 0.7412 - val_loss: 0.6424 - val_acc: 0.7361\n",
      "Epoch 5/16\n",
      "40429/40429 [==============================] - 2s 55us/step - loss: 0.6356 - acc: 0.7386 - val_loss: 0.6447 - val_acc: 0.7358\n",
      "Epoch 6/16\n",
      "40429/40429 [==============================] - 2s 54us/step - loss: 0.6289 - acc: 0.7401 - val_loss: 0.6453 - val_acc: 0.7350\n",
      "Epoch 7/16\n",
      "40429/40429 [==============================] - 2s 54us/step - loss: 0.6164 - acc: 0.7435 - val_loss: 0.6443 - val_acc: 0.7334\n",
      "Epoch 8/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.6063 - acc: 0.7470 - val_loss: 0.6446 - val_acc: 0.7337\n",
      "Epoch 9/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.6157 - acc: 0.7459 - val_loss: 0.6457 - val_acc: 0.7353\n",
      "Epoch 10/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.6043 - acc: 0.7494 - val_loss: 0.6486 - val_acc: 0.7316\n",
      "Epoch 11/16\n",
      "40429/40429 [==============================] - 2s 55us/step - loss: 0.5904 - acc: 0.7558 - val_loss: 0.6515 - val_acc: 0.7334\n",
      "Epoch 12/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.5785 - acc: 0.7605 - val_loss: 0.6517 - val_acc: 0.7313\n",
      "Epoch 13/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.5892 - acc: 0.7547 - val_loss: 0.6525 - val_acc: 0.7360\n",
      "Epoch 14/16\n",
      "40429/40429 [==============================] - 2s 58us/step - loss: 0.5802 - acc: 0.7596 - val_loss: 0.6551 - val_acc: 0.7327\n",
      "Epoch 15/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.5628 - acc: 0.7659 - val_loss: 0.6643 - val_acc: 0.7334\n",
      "Epoch 16/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.5442 - acc: 0.7743 - val_loss: 0.6632 - val_acc: 0.7315\n",
      "['Model-1.h5', 'Model-4.h5', 'Model-3.h5', 'Model-2.h5']\n",
      "35/35 [==============================] - 0s 4ms/step\n",
      "35/35 [==============================] - 0s 55us/step\n",
      "35/35 [==============================] - 0s 54us/step\n",
      "35/35 [==============================] - 0s 54us/step\n",
      "the kflod cv is :  0.7363535090976606\n",
      "Train on 40429 samples, validate on 8086 samples\n",
      "Epoch 1/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.6901 - acc: 0.7325 - val_loss: 0.6569 - val_acc: 0.7341\n",
      "Epoch 2/16\n",
      "40429/40429 [==============================] - 2s 57us/step - loss: 0.6479 - acc: 0.7363 - val_loss: 0.6510 - val_acc: 0.7356\n",
      "Epoch 3/16\n",
      "40429/40429 [==============================] - 2s 58us/step - loss: 0.6342 - acc: 0.7386 - val_loss: 0.6501 - val_acc: 0.7340\n",
      "Epoch 4/16\n",
      "40429/40429 [==============================] - 2s 60us/step - loss: 0.6271 - acc: 0.7410 - val_loss: 0.6509 - val_acc: 0.7341\n",
      "Epoch 5/16\n",
      "40429/40429 [==============================] - 2s 59us/step - loss: 0.6325 - acc: 0.7395 - val_loss: 0.6526 - val_acc: 0.7362\n",
      "Epoch 6/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.6257 - acc: 0.7415 - val_loss: 0.6520 - val_acc: 0.7332\n",
      "Epoch 7/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.6147 - acc: 0.7454 - val_loss: 0.6534 - val_acc: 0.7342\n",
      "Epoch 8/16\n",
      "40429/40429 [==============================] - 2s 55us/step - loss: 0.6063 - acc: 0.7477 - val_loss: 0.6529 - val_acc: 0.7339\n",
      "Epoch 9/16\n",
      "40429/40429 [==============================] - 2s 54us/step - loss: 0.6153 - acc: 0.7477 - val_loss: 0.6545 - val_acc: 0.7346\n",
      "Epoch 10/16\n",
      "40429/40429 [==============================] - 2s 54us/step - loss: 0.6052 - acc: 0.7503 - val_loss: 0.6552 - val_acc: 0.7331\n",
      "Epoch 11/16\n",
      "40429/40429 [==============================] - 2s 55us/step - loss: 0.5918 - acc: 0.7553 - val_loss: 0.6577 - val_acc: 0.7331\n",
      "Epoch 12/16\n",
      "40429/40429 [==============================] - 2s 55us/step - loss: 0.5793 - acc: 0.7599 - val_loss: 0.6593 - val_acc: 0.7324\n",
      "Epoch 13/16\n",
      "40429/40429 [==============================] - 2s 58us/step - loss: 0.5921 - acc: 0.7559 - val_loss: 0.6585 - val_acc: 0.7310\n",
      "Epoch 14/16\n",
      "40429/40429 [==============================] - 2s 56us/step - loss: 0.5826 - acc: 0.7596 - val_loss: 0.6637 - val_acc: 0.7304\n",
      "Epoch 15/16\n",
      "40429/40429 [==============================] - 2s 59us/step - loss: 0.5667 - acc: 0.7652 - val_loss: 0.6667 - val_acc: 0.7308\n",
      "Epoch 16/16\n",
      "40429/40429 [==============================] - 2s 58us/step - loss: 0.5512 - acc: 0.7715 - val_loss: 0.6696 - val_acc: 0.7324\n",
      "['Model-1.h5', 'Model-4.h5', 'Model-3.h5', 'Model-2.h5']\n",
      "35/35 [==============================] - 0s 4ms/step\n",
      "35/35 [==============================] - 0s 57us/step\n",
      "35/35 [==============================] - 0s 58us/step\n",
      "35/35 [==============================] - 0s 59us/step\n",
      "the kflod cv is :  0.7340346534653466\n",
      "Train on 40430 samples, validate on 8085 samples\n",
      "Epoch 1/16\n",
      "40430/40430 [==============================] - 3s 62us/step - loss: 0.6920 - acc: 0.7295 - val_loss: 0.6435 - val_acc: 0.7393\n",
      "Epoch 2/16\n",
      "40430/40430 [==============================] - 2s 59us/step - loss: 0.6516 - acc: 0.7359 - val_loss: 0.6345 - val_acc: 0.7406\n",
      "Epoch 3/16\n",
      "40430/40430 [==============================] - 2s 59us/step - loss: 0.6396 - acc: 0.7372 - val_loss: 0.6330 - val_acc: 0.7409\n",
      "Epoch 4/16\n",
      "40430/40430 [==============================] - 2s 61us/step - loss: 0.6320 - acc: 0.7387 - val_loss: 0.6325 - val_acc: 0.7416\n",
      "Epoch 5/16\n",
      "40430/40430 [==============================] - 2s 59us/step - loss: 0.6369 - acc: 0.7385 - val_loss: 0.6322 - val_acc: 0.7409\n",
      "Epoch 6/16\n",
      "40430/40430 [==============================] - 2s 58us/step - loss: 0.6299 - acc: 0.7378 - val_loss: 0.6331 - val_acc: 0.7391\n",
      "Epoch 7/16\n",
      "40430/40430 [==============================] - 2s 56us/step - loss: 0.6185 - acc: 0.7431 - val_loss: 0.6347 - val_acc: 0.7409\n",
      "Epoch 8/16\n",
      "40430/40430 [==============================] - 2s 58us/step - loss: 0.6085 - acc: 0.7446 - val_loss: 0.6335 - val_acc: 0.7411\n",
      "Epoch 9/16\n",
      "40430/40430 [==============================] - 2s 60us/step - loss: 0.6181 - acc: 0.7425 - val_loss: 0.6346 - val_acc: 0.7394\n",
      "Epoch 10/16\n",
      "40430/40430 [==============================] - 2s 59us/step - loss: 0.6093 - acc: 0.7464 - val_loss: 0.6360 - val_acc: 0.7377\n",
      "Epoch 11/16\n",
      "40430/40430 [==============================] - 2s 60us/step - loss: 0.5956 - acc: 0.7529 - val_loss: 0.6401 - val_acc: 0.7363\n",
      "Epoch 12/16\n",
      "40430/40430 [==============================] - 2s 59us/step - loss: 0.5849 - acc: 0.7575 - val_loss: 0.6389 - val_acc: 0.7368\n",
      "Epoch 13/16\n",
      "40430/40430 [==============================] - 2s 58us/step - loss: 0.5955 - acc: 0.7530 - val_loss: 0.6434 - val_acc: 0.7382\n",
      "Epoch 14/16\n",
      "40430/40430 [==============================] - 2s 60us/step - loss: 0.5858 - acc: 0.7591 - val_loss: 0.6488 - val_acc: 0.7384\n",
      "Epoch 15/16\n",
      "40430/40430 [==============================] - 2s 57us/step - loss: 0.5687 - acc: 0.7643 - val_loss: 0.6463 - val_acc: 0.7320\n",
      "Epoch 16/16\n",
      "40430/40430 [==============================] - 2s 59us/step - loss: 0.5540 - acc: 0.7699 - val_loss: 0.6478 - val_acc: 0.7344\n",
      "['Model-1.h5', 'Model-4.h5', 'Model-3.h5', 'Model-2.h5']\n",
      "35/35 [==============================] - 0s 5ms/step\n",
      "35/35 [==============================] - 0s 56us/step\n",
      "35/35 [==============================] - 0s 58us/step\n",
      "35/35 [==============================] - 0s 55us/step\n",
      "the kflod cv is :  0.7402870576589953\n",
      "total scores is  0.7374167982446459\n"
     ]
    }
   ],
   "source": [
    "predicts = stacking_pseudo(train, train_y, test, predicts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_result(predict):\n",
    "    with open('../data/pickle.pkl', 'wb') as f:\n",
    "        pickle.dump(predict, f)\n",
    "\n",
    "    results = predict\n",
    "    count_zero = 0\n",
    "    count_two = 0\n",
    "    count_one = 0\n",
    "    with open(TEXT_X, 'r', encoding='utf-8') as f, open('../data/' + 'result.txt', 'w', encoding='utf-8') as d:\n",
    "        lines = f.readlines()\n",
    "        for count, line in enumerate(lines):\n",
    "            line = line.strip()\n",
    "            line = line.split('\\t')\n",
    "            id = line[0]\n",
    "            flag = np.argmax(results[count])\n",
    "            if flag == 1:\n",
    "                count_one += 1\n",
    "            elif flag == 0:\n",
    "                count_zero += 1\n",
    "            elif flag == 2:\n",
    "                count_two += 1\n",
    "            d.write(id + '\\t' + str(flag) + '\\t' + 'NULL' + '\\t' + 'NULL')\n",
    "            d.write('\\n')\n",
    "    print(count_one)\n",
    "    print(count_one / len(results))\n",
    "    print(count_zero / len(results))\n",
    "    print(count_two / len(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "0.08571428571428572\n",
      "0.6571428571428571\n",
      "0.2571428571428571\n"
     ]
    }
   ],
   "source": [
    "save_result(predicts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
